{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Layers : Fundamental blocks of Neural Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch.nn import Linear, ReLU\n",
    "import torch.nn as nn\n",
    "import numpy as np\n",
    "from torch.autograd import Variable\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 0.6173  0.1456 -0.3605 -0.0123 -0.9308\n",
       "[torch.FloatTensor of size 1x5]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myLayer = Linear(in_features=10,out_features=5,bias=True)\n",
    "inp = Variable(torch.randn(1,10))\n",
    "myLayer = Linear(in_features=10,out_features=5,bias=True) \n",
    "myLayer(inp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "-0.1077  0.2764 -0.0719 -0.1307  0.1424 -0.2288 -0.1666  0.1439  0.3042 -0.2801\n",
       "-0.1775  0.2681 -0.0646 -0.1185 -0.3148 -0.1193  0.0045  0.1840 -0.2018  0.0299\n",
       " 0.1641  0.0864 -0.0993  0.0208  0.0890  0.1002 -0.1926 -0.0080 -0.1631 -0.0869\n",
       " 0.2058  0.1669  0.1589 -0.2068 -0.2168  0.1223 -0.2303 -0.1564 -0.1564 -0.2926\n",
       "-0.3048 -0.2558 -0.2374 -0.0940 -0.0869  0.1781 -0.0560 -0.2655  0.1365  0.1312\n",
       "[torch.FloatTensor of size 5x10]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myLayer.weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       " 0.3093\n",
       "-0.1218\n",
       "-0.2188\n",
       " 0.0249\n",
       "-0.1592\n",
       "[torch.FloatTensor of size 5]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myLayer.bias"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Stacking Linear layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 0.4065 -0.7448\n",
       "[torch.FloatTensor of size 1x2]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myLayer1 = Linear(10,5)\n",
    "myLayer2 = Linear(5,2)\n",
    "myLayer2(myLayer1(inp))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### PyTorch Non-linear Activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 1  2  0  0\n",
       "[torch.FloatTensor of size 1x4]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_data = Variable(torch.Tensor([[1,2,-1,-1]])) \n",
    "myRelu = ReLU()\n",
    "myRelu(sample_data)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Variable containing:\n",
       " 1  2  0  0\n",
       "[torch.FloatTensor of size 1x4]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "sample_data = Variable(torch.Tensor([[1,2,-1,-1]])) \n",
    "f = F.relu(sample_data) # Much simpler.\n",
    "f"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Neural Network "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class MyFirstNetwork(nn.Module):\n",
    "    def __init__(self,input_size,hidden_size,output_size):\n",
    "        super(MyFirstNetwork,self).__init__() \n",
    "        self.layer1 = nn.Linear(input_size,hidden_size) \n",
    "        self.layer2 = nn.Linear(hidden_size,output_size)\n",
    "    def __forward__(self,input): \n",
    "        out = self.layer1(input) \n",
    "        out = nn.ReLU(out)\n",
    "        out = self.layer2(out) \n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = nn.MSELoss()\n",
    "input = Variable(torch.randn(3, 5), requires_grad=True) \n",
    "target = Variable(torch.randn(3, 5))\n",
    "output = loss(input, target)\n",
    "output.backward()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cross_entropy(true_label, prediction):\n",
    "    if true_label == 1:\n",
    "        return -log(prediction)\n",
    "    else:\n",
    "        return -log(1 - prediction)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "loss = nn.CrossEntropyLoss()\n",
    "input = Variable(torch.randn(3, 5), requires_grad=True) \n",
    "target = Variable(torch.LongTensor(3).random_(5)) \n",
    "output = loss(input, target)\n",
    "output.backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for demo\n",
    "import torch.optim as optim\n",
    "optimizer = optim.SGD(model.parameters(), lr = 0.01)\n",
    "for input, target in dataset:\n",
    "    optimizer.zero_grad()\n",
    "    output = model(input)\n",
    "    loss = loss_fn(output, target)\n",
    "    loss.backward()\n",
    "    optimizer.step()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
